#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
湖南省公共资源交易中心爬虫
爬取建设工程招标计划列表和详情数据，保存为Excel文件
"""

import requests
import json
import pandas as pd
import time
import logging
from datetime import datetime, timedelta
from typing import List, Dict, Any
import os
from config import OUTPUT_CONFIG

# 配置日志
logging.basicConfig(
    level=logging.INFO,
    format='%(asctime)s - %(levelname)s - %(message)s',
    handlers=[
        logging.FileHandler('crawler.log', encoding='utf-8'),
        logging.StreamHandler()
    ]
)
logger = logging.getLogger(__name__)

class HNSGGZYCrawler:
    def __init__(self):
        self.base_url = "https://www.hnsggzy.com"
        self.session = requests.Session()
        
        # 设置请求头
        self.headers = {
            "Accept": "application/json, text/plain, */*",
            "Accept-Language": "zh-CN,zh;q=0.9",
            "Connection": "keep-alive",
            "Referer": "https://www.hnsggzy.com/",
            "Sec-Fetch-Dest": "empty",
            "Sec-Fetch-Mode": "cors",
            "Sec-Fetch-Site": "same-origin",
            "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/140.0.0.0 Safari/537.36",
            "sec-ch-ua": '"Chromium";v="140", "Not=A?Brand";v="24", "Google Chrome";v="140"',
            "sec-ch-ua-mobile": "?0",
            "sec-ch-ua-platform": '"Windows"'
        }
        
        # 设置cookies（需要根据实际情况更新）
        self.cookies = {
            "JSESSIONID": "e24a7a43-c642-4726-901b-ab09dac5b1e6",
            "SESSION": "ZTI0YTdhNDMtYzY0Mi00NzI2LTkwMWItYWIwOWRhYzViMWU2"
        }
        
        self.session.headers.update(self.headers)
        self.session.cookies.update(self.cookies)
    
    def get_list_data(self, current: int = 1, size: int = 10, 
                     start_date: str = None, end_date: str = None) -> Dict[str, Any]:
        """
        获取列表数据
        
        Args:
            current: 当前页码
            size: 每页大小
            start_date: 开始日期 (格式: 2025-09-25 00:00:00)
            end_date: 结束日期 (格式: 2025-09-26 23:59:59)
        
        Returns:
            列表数据字典
        """
        url = f"{self.base_url}/tradeApi/constructionTender/ListConstructionTenderPlan"
        
        params = {
            "current": current,
            "size": size,
            "descs": "noticeSendDate"
        }
        
        if start_date:
            params["noticeSendTimeStart"] = start_date
        if end_date:
            params["noticeSendTimeEnd"] = end_date
        
        try:
            logger.info(f"正在获取列表数据 - 页码: {current}, 每页: {size}")
            response = self.session.get(url, params=params, timeout=30)
            response.raise_for_status()
            
            data = response.json()
            # 检查响应结构
            if data.get('code') == 200 and 'data' in data:
                actual_data = data['data']
                logger.info(f"列表数据获取成功 - 总数: {actual_data.get('total', 0)}")
                return actual_data
            else:
                logger.error(f"API响应错误: {data.get('msg', '未知错误')}")
                return {}
            
        except requests.exceptions.RequestException as e:
            logger.error(f"获取列表数据失败: {e}")
            return {}
        except json.JSONDecodeError as e:
            logger.error(f"解析列表数据失败: {e}")
            return {}
    
    def get_detail_data(self, item_id: str) -> Dict[str, Any]:
        """
        获取详情数据
        
        Args:
            item_id: 项目ID
        
        Returns:
            详情数据字典
        """
        url = f"{self.base_url}/tradeApi/constructionTender/getConstructionTenderPlanById"
        params = {"id": item_id}
        
        try:
            logger.info(f"正在获取详情数据 - ID: {item_id}")
            response = self.session.get(url, params=params, timeout=30)
            response.raise_for_status()
            
            data = response.json()
            # 检查响应结构
            if data.get('code') == 200 and 'data' in data:
                actual_data = data['data']
                logger.info(f"详情数据获取成功 - ID: {item_id}")
                return actual_data
            else:
                logger.error(f"详情API响应错误 - ID: {item_id}, 错误: {data.get('msg', '未知错误')}")
                return {}
            
        except requests.exceptions.RequestException as e:
            logger.error(f"获取详情数据失败 - ID: {item_id}, 错误: {e}")
            return {}
        except json.JSONDecodeError as e:
            logger.error(f"解析详情数据失败 - ID: {item_id}, 错误: {e}")
            return {}
    
    def get_all_list_data(self, start_date: str = None, end_date: str = None, 
                         max_pages: int = None) -> List[Dict[str, Any]]:
        """
        获取所有列表数据
        
        Args:
            start_date: 开始日期
            end_date: 结束日期
            max_pages: 最大页数限制
        
        Returns:
            所有列表数据
        """
        all_data = []
        current = 1
        size = 50  # 每页获取更多数据
        
        while True:
            if max_pages and current > max_pages:
                break
                
            data = self.get_list_data(current, size, start_date, end_date)
            
            if not data or 'records' not in data:
                logger.warning(f"第 {current} 页数据为空，停止获取")
                break
            
            records = data['records']
            if not records:
                logger.info(f"第 {current} 页无数据，停止获取")
                break
            
            all_data.extend(records)
            logger.info(f"已获取 {len(all_data)} 条列表数据")
            
            # 检查是否还有更多页
            total = data.get('total', 0)
            if len(all_data) >= total:
                break
            
            current += 1
            time.sleep(1)  # 避免请求过快
        
        logger.info(f"列表数据获取完成，共 {len(all_data)} 条")
        return all_data
    
    def get_all_details(self, list_data: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
        """
        获取所有详情数据
        
        Args:
            list_data: 列表数据
        
        Returns:
            所有详情数据
        """
        all_details = []
        total = len(list_data)
        
        for i, item in enumerate(list_data, 1):
            item_id = item.get('id')
            if not item_id:
                logger.warning(f"第 {i} 条数据缺少ID，跳过")
                continue
            
            detail = self.get_detail_data(item_id)
            if detail:
                # 合并列表数据和详情数据
                combined_data = {**item, **detail}
                all_details.append(combined_data)
            
            logger.info(f"详情数据获取进度: {i}/{total}")
            time.sleep(1)  # 避免请求过快
        
        logger.info(f"详情数据获取完成，共 {len(all_details)} 条")
        return all_details
    
    def save_to_excel(self, data: List[Dict[str, Any]], filename: str = None) -> str:
        """
        保存数据到Excel文件
        
        Args:
            data: 要保存的数据
            filename: 文件名
        
        Returns:
            保存的文件路径
        """
        if not data:
            logger.warning("没有数据需要保存")
            return ""
        
        if not filename:
            timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
            filename = f"hnsggzy_data_{timestamp}.xlsx"
        
        try:
            # 创建DataFrame
            df = pd.DataFrame(data)
            
            # 保存到Excel
            df.to_excel(filename, index=False, engine='openpyxl')
            
            logger.info(f"数据已保存到Excel: {filename}")
            logger.info(f"共保存 {len(data)} 条数据，{len(df.columns)} 个字段")
            
            return filename
            
        except Exception as e:
            logger.error(f"保存Excel文件失败: {e}")
            return ""
    
    def save_to_json(self, data: List[Dict[str, Any]], filename: str = None) -> str:
        """
        保存数据到JSON文件
        
        Args:
            data: 要保存的数据
            filename: 文件名
        
        Returns:
            保存的文件路径
        """
        if not data:
            logger.warning("没有数据需要保存")
            return ""
        
        if not filename:
            timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
            filename = f"hnsggzy_data_{timestamp}.json"
        
        try:
            # 保存到JSON文件，使用配置文件中的设置
            json_config = OUTPUT_CONFIG.get("output_formats", {})
            encoding = json_config.get("json_encoding", "utf-8")
            indent = json_config.get("json_indent", 2)
            ensure_ascii = json_config.get("json_ensure_ascii", False)
            
            with open(filename, 'w', encoding=encoding) as f:
                json.dump(data, f, ensure_ascii=ensure_ascii, indent=indent)
            
            logger.info(f"数据已保存到JSON: {filename}")
            logger.info(f"共保存 {len(data)} 条数据")
            
            return filename
            
        except Exception as e:
            logger.error(f"保存JSON文件失败: {e}")
            return ""
    
    def crawl_all_data(self, start_date: str = None, end_date: str = None, 
                      max_pages: int = None) -> Dict[str, str]:
        """
        爬取所有数据并同时保存为JSON和Excel格式
        
        Args:
            start_date: 开始日期
            end_date: 结束日期
            max_pages: 最大页数限制
        
        Returns:
            包含JSON和Excel文件路径的字典
        """
        logger.info("开始爬取数据...")
        
        # 获取列表数据
        list_data = self.get_all_list_data(start_date, end_date, max_pages)
        if not list_data:
            logger.error("未获取到列表数据")
            return {"json": "", "excel": ""}
        
        # 获取详情数据
        all_data = self.get_all_details(list_data)
        if not all_data:
            logger.error("未获取到详情数据")
            return {"json": "", "excel": ""}
        
        # 生成统一的时间戳用于文件名
        timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
        filename_prefix = OUTPUT_CONFIG.get("filename_prefix", "hnsggzy_data")
        
        # 根据配置决定保存哪些格式
        output_formats = OUTPUT_CONFIG.get("output_formats", {})
        save_json = output_formats.get("save_json", True)
        save_excel = output_formats.get("save_excel", True)
        
        result = {"json": "", "excel": ""}
        
        if save_json:
            json_filename = f"{filename_prefix}_{timestamp}.json"
            json_path = self.save_to_json(all_data, json_filename)
            result["json"] = json_path
        
        if save_excel:
            excel_filename = f"{filename_prefix}_{timestamp}.xlsx"
            excel_path = self.save_to_excel(all_data, excel_filename)
            result["excel"] = excel_path
        
        logger.info("数据爬取完成！")
        if result["json"]:
            logger.info(f"JSON文件: {result['json']}")
        if result["excel"]:
            logger.info(f"Excel文件: {result['excel']}")
        
        return result


def main():
    """主函数"""
    crawler = HNSGGZYCrawler()
    
    # 设置日期范围（可以根据需要修改）
    # 使用当前月份的数据
    start_date = "2024-12-01 00:00:00"
    end_date = "2024-12-31 23:59:59"
    
    # 限制页数（测试时使用，正式运行时可以设为None）
    max_pages = 5
    
    # 开始爬取
    result = crawler.crawl_all_data(start_date, end_date, max_pages)
    
    if result["json"] and result["excel"]:
        print(f"\n数据爬取完成！")
        print(f"JSON文件已保存为: {result['json']}")
        print(f"Excel文件已保存为: {result['excel']}")
    else:
        print("\n数据爬取失败！")


if __name__ == "__main__":
    main()
